In [1]:
import pylearn2.utils
import pylearn2.config
import theano
import neukrill_net.dense_dataset
import neukrill_net.utils
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
import holoviews as hl
%load_ext holoviews.ipython
import sklearn.metrics
In [37]:
m = pylearn2.utils.serial.load(
"/disk/scratch/neuroglycerin/models/replicate_8aug.pkl")
In [38]:
print(m)
In [6]:
nll_channels = [c for c in m.monitor.channels.keys() if 'nll' in c]
In [7]:
import neukrill_net.plotting as pl
In [39]:
pl.monitor_channels(m, nll_channels, x_axis='time')
Out[39]:
In [40]:
c = m.monitor.channels['valid_y_nll']
In [42]:
plt.plot(c.val_record[200:])
Out[42]:
In [11]:
24*60*60
Out[11]:
In [13]:
%run ~/repos/pylearn2/pylearn2/scripts/print_monitor.py /disk/scratch/neuroglycerin/models/replicate_8aug_recent.pkl
In [14]:
%run ~/repos/pylearn2/pylearn2/scripts/print_monitor.py /disk/scratch/neuroglycerin/models/alexnet_based_40aug.pkl
In [16]:
60084.537869/(17*60)
Out[16]:
In [17]:
m = pylearn2.utils.serial.load(
"/disk/scratch/neuroglycerin/models/alexnet_based_40aug.pkl")
In [18]:
pl.monitor_channels(m, nll_channels, x_axis='time')
Out[18]:
In [22]:
pl.monitor_channels(m,['learning_rate'], x_axis='epoch')
Out[22]:
In [43]:
%run ~/repos/pylearn2/pylearn2/scripts/print_monitor.py /disk/scratch/neuroglycerin/models/alexnet_based_40aug.pkl
In [23]:
m = pylearn2.utils.serial.load(
"/disk/scratch/neuroglycerin/models/resume_40aug_recent.pkl")
In [24]:
pl.monitor_channels(m, nll_channels, x_axis='time')
Out[24]:
In [25]:
pl.monitor_channels(m,['learning_rate'], x_axis='epoch')
Out[25]:
Looks like something about the model is definitely broken.
In [26]:
m = pylearn2.utils.serial.load(
"/disk/scratch/neuroglycerin/models/replicate_8aug_lrschedule.pkl")
In [27]:
pl.monitor_channels(m, nll_channels, x_axis='time')
Out[27]:
Validation score failed to reach anywhere what it manages above, with only a slight change to the learning rate schedule. Could be something else is going wrong somewhere. Could try rolling back changes to the state we were at when running replicate 8aug originally.
In [44]:
m = pylearn2.utils.serial.load(
"/disk/scratch/neuroglycerin/models/resume_40aug.pkl")
In [46]:
c = m.monitor.channels['train_y_nll']
plt.plot(c.val_record)
Out[46]:
In [47]:
pl.monitor_channels(m, m.monitor.channels.keys(), x_axis="epoch")
Out[47]:
In [48]:
m = pylearn2.utils.serial.load(
"/disk/scratch/neuroglycerin/models/resume_40aug.pkl")
In [51]:
pl.monitor_channels(m, ['train_y_nll'], x_axis='epoch')pwd
Out[51]: